{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\thushan\\documents\\python_virtualenvs\\tensorflow_venv\\lib\\site-packages\\h5py\\__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "from tensorflow.python.keras.models import Sequential\n",
    "from tensorflow.python.keras.layers import Conv2D,Dense,MaxPool2D,BatchNormalization,Flatten\n",
    "from tensorflow.python.keras import backend as K\n",
    "\n",
    "# Required for Data downaload and preparation\n",
    "import struct\n",
    "import gzip\n",
    "import os\n",
    "from six.moves.urllib.request import urlretrieve"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Lolading Data\n",
    "\n",
    "Here we download (if needed) the MNIST dataset and, perform reshaping and normalization. Also we conver the labels to one hot encoded vectors."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found and verified train-images-idx3-ubyte.gz\n",
      "Found and verified train-labels-idx1-ubyte.gz\n",
      "Found and verified t10k-images-idx3-ubyte.gz\n",
      "Found and verified t10k-labels-idx1-ubyte.gz\n",
      "\n",
      "Reading files train-images-idx3-ubyte.gz and train-labels-idx1-ubyte.gz\n",
      "60000 28 28\n",
      "(Images) Returned a tensor of shape  (60000, 28, 28, 1)\n",
      "(Labels) Returned a tensor of shape: 60000\n",
      "Sample labels:  [5 0 4 1 9 2 1 3 1 4]\n",
      "\n",
      "Reading files t10k-images-idx3-ubyte.gz and t10k-labels-idx1-ubyte.gz\n",
      "10000 28 28\n",
      "(Images) Returned a tensor of shape  (10000, 28, 28, 1)\n",
      "(Labels) Returned a tensor of shape: 10000\n",
      "Sample labels:  [7 2 1 0 4 1 4 9 5 9]\n",
      "\n",
      "Train size:  60000\n",
      "\n",
      "Test size:  10000\n"
     ]
    }
   ],
   "source": [
    "def maybe_download(url, filename, expected_bytes, force=False):\n",
    "  \"\"\"Download a file if not present, and make sure it's the right size.\"\"\"\n",
    "  if force or not os.path.exists(filename):\n",
    "    print('Attempting to download:', filename) \n",
    "    filename, _ = urlretrieve(url + filename, filename)\n",
    "    print('\\nDownload Complete!')\n",
    "  statinfo = os.stat(filename)\n",
    "  if statinfo.st_size == expected_bytes:\n",
    "    print('Found and verified', filename)\n",
    "  else:\n",
    "    raise Exception(\n",
    "      'Failed to verify ' + filename + '. Can you get to it with a browser?')\n",
    "  return filename\n",
    "\n",
    "\n",
    "def read_mnist(fname_img, fname_lbl, one_hot=False):\n",
    "    print('\\nReading files %s and %s'%(fname_img, fname_lbl))\n",
    "    \n",
    "    # Processing images\n",
    "    with gzip.open(fname_img) as fimg:        \n",
    "        magic, num, rows, cols = struct.unpack(\">IIII\", fimg.read(16))\n",
    "        print(num,rows,cols)\n",
    "        img = (np.frombuffer(fimg.read(num*rows*cols), dtype=np.uint8).reshape(num, rows, cols,1)).astype(np.float32)\n",
    "        print('(Images) Returned a tensor of shape ',img.shape)\n",
    "        \n",
    "        #img = (img - np.mean(img)) /np.std(img)\n",
    "        img *= 1.0 / 255.0\n",
    "    \n",
    "    # Processing labels\n",
    "    with gzip.open(fname_lbl) as flbl:\n",
    "        # flbl.read(8) reads upto 8 bytes\n",
    "        magic, num = struct.unpack(\">II\", flbl.read(8))               \n",
    "        lbl = np.frombuffer(flbl.read(num), dtype=np.int8)\n",
    "        if one_hot:\n",
    "            one_hot_lbl = np.zeros(shape=(num,10),dtype=np.float32)\n",
    "            one_hot_lbl[np.arange(num),lbl] = 1.0\n",
    "        print('(Labels) Returned a tensor of shape: %s'%lbl.shape)\n",
    "        print('Sample labels: ',lbl[:10])\n",
    "    \n",
    "    if not one_hot:\n",
    "        return img, lbl\n",
    "    else:\n",
    "        return img, one_hot_lbl\n",
    "    \n",
    "    \n",
    "# Download data if needed\n",
    "url = 'http://yann.lecun.com/exdb/mnist/'\n",
    "# training data\n",
    "maybe_download(url,'train-images-idx3-ubyte.gz',9912422)\n",
    "maybe_download(url,'train-labels-idx1-ubyte.gz',28881)\n",
    "# testing data\n",
    "maybe_download(url,'t10k-images-idx3-ubyte.gz',1648877)\n",
    "maybe_download(url,'t10k-labels-idx1-ubyte.gz',4542)\n",
    "\n",
    "# Read the training and testing data \n",
    "train_inputs, train_labels = read_mnist('train-images-idx3-ubyte.gz', 'train-labels-idx1-ubyte.gz',True)\n",
    "test_inputs, test_labels = read_mnist('t10k-images-idx3-ubyte.gz', 't10k-labels-idx1-ubyte.gz',True)\n",
    "\n",
    "\n",
    "print('\\nTrain size: ', train_inputs.shape[0])\n",
    "print('\\nTest size: ', test_inputs.shape[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Generators for MNIST\n",
    "\n",
    "Here we have the logic to iterate through each training, validation and testing datasets, in `batch_size` size strides."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_index, test_index = 0,0\n",
    "\n",
    "def get_train_batch(images, labels, batch_size):\n",
    "    global train_index\n",
    "    batch = images[train_index:train_index+batch_size,:,:,:], labels[train_index:train_index+batch_size,:]\n",
    "    train_index = (train_index + batch_size)%(images.shape[0] - batch_size)\n",
    "    return batch\n",
    "\n",
    "\n",
    "def get_test_batch(images, labels, batch_size):\n",
    "    global test_index\n",
    "    batch = images[test_index:test_index+batch_size,:,:,:], labels[test_index:test_index+batch_size,:]\n",
    "    test_index = (test_index + batch_size)%(images.shape[0] - batch_size)\n",
    "    return batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "config = tf.ConfigProto(allow_soft_placement=True)\n",
    "# Good practice to use this to avoid any surprising errors thrown by TensorFlow\n",
    "config.gpu_options.allow_growth = True \n",
    "config.gpu_options.per_process_gpu_memory_fraction = 0.9 # Making sure Tensorflow doesn't overflow the GPU\n",
    "sess = tf.Session(config=config)\n",
    "K.set_session(sess)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Define a sequential model\n",
    "model = Sequential()\n",
    "\n",
    "# Added a convolution layer\n",
    "model.add(Conv2D(32, 3, activation='relu', input_shape=[28, 28, 1]))\n",
    "\n",
    "# Add a max pool lyer\n",
    "model.add(MaxPool2D())\n",
    "\n",
    "# Add a batch norm layer\n",
    "model.add(BatchNormalization())\n",
    "\n",
    "# Convolution layer\n",
    "model.add(Conv2D(64, 3, activation='relu'))\n",
    "# Max pool layer\n",
    "model.add(MaxPool2D())\n",
    "# Add a batch norm layer\n",
    "model.add(BatchNormalization())\n",
    "\n",
    "# More convolution, max pool, batch norm\n",
    "model.add(Conv2D(128, 3, activation='relu'))\n",
    "model.add(MaxPool2D())\n",
    "model.add(BatchNormalization())\n",
    "\n",
    "model.add(Flatten())\n",
    "\n",
    "model.add(Dense(256, activation='relu'))\n",
    "model.add(Dense(10, activation='softmax'))\n",
    "\n",
    "model.compile(\n",
    "    optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy']\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training for epoch:  0\n",
      "Epoch 1/1\n",
      "60000/60000 [==============================] - 13s 210us/step - loss: 0.1245 - acc: 0.9628\n",
      "10000/10000 [==============================] - 1s 67us/step\n",
      "\tEpoch ( 0 ) Test accuracy:  0.9757000070810318\n",
      "Training for epoch:  1\n",
      "Epoch 1/1\n",
      "60000/60000 [==============================] - 11s 182us/step - loss: 0.0452 - acc: 0.9859\n",
      "10000/10000 [==============================] - 1s 55us/step\n",
      "\tEpoch ( 1 ) Test accuracy:  0.9827000075578689\n",
      "Training for epoch:  2\n",
      "Epoch 1/1\n",
      "60000/60000 [==============================] - 11s 186us/step - loss: 0.0294 - acc: 0.9910\n",
      "10000/10000 [==============================] - 1s 56us/step\n",
      "\tEpoch ( 2 ) Test accuracy:  0.9847000110149383\n",
      "Training for epoch:  3\n",
      "Epoch 1/1\n",
      "60000/60000 [==============================] - 11s 184us/step - loss: 0.0255 - acc: 0.9918\n",
      "10000/10000 [==============================] - 1s 56us/step\n",
      "\tEpoch ( 3 ) Test accuracy:  0.9859000080823899\n",
      "Training for epoch:  4\n",
      "Epoch 1/1\n",
      "60000/60000 [==============================] - 11s 182us/step - loss: 0.0197 - acc: 0.9937\n",
      "10000/10000 [==============================] - 1s 56us/step\n",
      "\tEpoch ( 4 ) Test accuracy:  0.9872000062465668\n"
     ]
    }
   ],
   "source": [
    "n_epochs = 5 # Number of epochs the training runs for\n",
    "\n",
    "n_train = 55000\n",
    "n_test = 10000\n",
    "\n",
    "batch_size = 100\n",
    "\n",
    "x_train, y_train = train_inputs, train_labels\n",
    "      \n",
    "x_test, y_test = test_inputs, test_labels\n",
    "    \n",
    "for epoch in range(n_epochs):\n",
    "\n",
    "    print('Training for epoch: ',epoch)    \n",
    "    \n",
    "    # Training for a single epoch\n",
    "    model.fit(x_train, y_train, batch_size = batch_size)\n",
    "    \n",
    "    # Testing phase\n",
    "    # Returns a list where first item is loss and second is accuracy\n",
    "    test_acc = model.evaluate(x_test, y_test, batch_size=batch_size)  \n",
    "    print('\\tEpoch (', epoch ,') Test accuracy: ',test_acc[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
